R Notebook

This is an R Markdown Notebook. When you execute code within the notebook, the results appear beneath the code.

Try executing this chunk by clicking the Run button within the chunk or by placing your cursor inside it and pressing Ctrl+Shift+Enter.

#BEGIN HERE—————-

library(data.table)
library(tidyverse)
library(lubridate)

file='/media/ad/01D6B57CFBE4DB20/1.Linux/Data/FQC/V04R-V04R-SQLData_3000_head.dat' 
#file='/media/ad/01D6B57CFBE4DB20/1.Linux/Data/FQC/V04R-V04R-SQLData_3000.dat'# 3.6 GB
#file<-'/media/ad/01D6B57CFBE4DB20/1.Linux/Data/FQC/V04R-V04R-SQLData_E_series.dat' # 2.9 GB crack session (ram?)
#file<-'/media/ad/01D6B57CFBE4DB20/1.Linux/Data/FQC/V04R-V04R-SQLData_Ric.dat' # 2.8 GB OK
data_head<-fread(file,fill=TRUE,nrows=5)
data_head

#install.packages('doParallel')
#Method 1
#library(doParallel)
#no_cores <- detectCores()
#print(no_cores)
#cl <- makeCluster(no_cores)  
#registerDoParallel(cl)  

#Method 2
#library(doParallel)
#cl <- makePSOCKcluster(5)  
#registerDoParallel(cl)  

# method 3
setDTthreads(threads = 0)
getDTthreads(verbose=TRUE) 

  omp_get_num_procs()            2
  R_DATATABLE_NUM_PROCS_PERCENT  unset (default 50)
  R_DATATABLE_NUM_THREADS        unset
  R_DATATABLE_THROTTLE           unset (default 1024)
  omp_get_thread_limit()         2147483647
  omp_get_max_threads()          2
  OMP_THREAD_LIMIT               unset
  OMP_NUM_THREADS                unset
  RestoreAfterFork               true
  data.table is using 2 threads with throttle==1024. See ?setDTthreads.
[1] 2

start.time <- Sys.time()

product_type_column='V3'
tryCatch(
  expr    = {dt1 <<- fread(file,fill=TRUE,select=c(product_type_column))},
  warning = function(w){
    cat('Warning: ', w$message, '\n\n');
    n_line <- as.numeric(gsub('Stopped early on line (\\d+)\\..*','\\1',w$message))
    if (!is.na(n_line)) {
      cat('Found ', n_line,'\n')
      dt1_part1 <- fread(file,fill=TRUE, nrows=n_line-1,select=c(product_type_column))
      dt1_part2 <- fread(file,fill=TRUE,skip=n_line,select=c(product_type_column))
      dt1 <<- rbind(dt1_part1, dt1_part2, fill=T)
    }
  },
  finally = cat("\nFinished. \n")
  #finally=.Call(CfreadCleanup)
);

Finished. 

end.time <- Sys.time()
time.taken <- end.time - start.time
time.taken 

Time difference of 0.2093291 secs

# No Parallel: 1.118 mins
# doParallel method 1: 1.097 mins --> run lan 2 la 58 or 52 s 
# doParallel method 2: 1.118 mins
# doParallel method 3: 51 s

#install.packages("pryr")
library(pryr)

Registered S3 method overwritten by 'pryr':
  method      from
  print.bytes Rcpp

Attaching package: ‘pryr’

The following objects are masked from ‘package:purrr’:

    compose, partial

The following object is masked from ‘package:data.table’:

    address

object_size(dt1) # 43.7 MB

405 kB

#product_type_column='V3'
#dt1<-fread(file,fill=TRUE,select=c(product_type_column))
dim(dt1)

[1] 50000     1

head(dt1)

tail(dt1)

class(dt1)

[1] "data.table" "data.frame"

product_all <- as.data.frame(table(dt1)) %>% 
  arrange(desc(Freq)) %>% 
  top_n(20,Freq)
product_all

product_all %>% 
  ggplot(aes(x= reorder(dt1, -Freq),y=Freq)) +
  geom_bar(stat="identity",fill="steelblue")+
  theme(axis.text.x=element_text(angle=90))

# Please choose product_type:
product_type='31AA015'
#product_type='31A007G_P'
# Please choose parameter column to show
parameter_column_name='V65'
parameter_column_value='V66'
# Default time column:
product_type_column='V3'
date_column='V6'

start.time <- Sys.time()
tryCatch(
  expr    = {dt2 <<- fread(file,fill=TRUE,select=c(product_type_column,date_column,parameter_column_name,parameter_column_value)) %>% 
    filter(V3 ==product_type)
  },
  warning = function(w){
    cat('Warning: ', w$message, '\n\n');
    n_line2 <- as.numeric(gsub('Stopped early on line (\\d+)\\..*','\\1',w$message))
    if (!is.na(n_line2)) {
      cat('Found ', n_line2,'\n')
      dt2_part1 <- fread(file,fill=TRUE, nrows=n_line2-1,select=c(product_type_column,date_column,parameter_column_name,parameter_column_value))
      dt2_part2 <- fread(file,fill=TRUE,skip=n_line2,select=c(product_type_column,date_column,
                                                          parameter_column_name,parameter_column_value))
      dt2 <<- rbind(dt2_part1, dt2_part2, fill=T)%>% 
      filter(V3 ==product_type)
    }
  },
  finally = cat("\nFinished. \n")
);

Finished. 

end.time <- Sys.time()
time.taken <- end.time - start.time
time.taken # 1.1207 mins , run lan 2: 46.15 s

Time difference of 0.1885769 secs

# Method 3: 43 s

object_size(dt2) # 60 MB

754 kB

env <- environment()
all_data_memory <-data.frame(
object = ls(env),
size = unlist(lapply(
  ls(env), function(x) {
  object.size(get(x, envir = env, inherits = FALSE))
}
)))
all_data_memory

#60 Megabytes (MB)  =   62,914,560 Bytes (B)

#Remove dt 1 to save ram
#rm(dt1)

dim(dt2)

[1] 6965    4

head(dt2)

tail(dt2)

colnames(dt2) <- c("product_type", "date","parameter_name","parameter_value")
dt2

print(dim(dt2))

[1] 6965    4

sl<-object.size(dt2)
print(sl, units = "auto", standard = "SI") 

754 kB

data_process <- dt2 %>%
  mutate( date_trans=mdy_hms(date), # must have mdy_hms for convert date time
          date_filter=as_date(date_trans),
         )
head(data_process)

tail(data_process)

print(dim(data_process))

[1] 6965    6

sl<-object.size(data_process)
print(sl, units = "auto", standard = "SI") 

866.5 kB

#rm(dt2)

min(data_process$date)

[1] "12/18/2017 23:31:1"

max(data_process$date)

[1] "12/27/2017 23:30:1"

data_process[which.max(data_process$date)]

NAs introduced by coercion

class(data_process$date_trans)

[1] "POSIXct" "POSIXt" 

class(data_process$date)

[1] "character"

min(data_process$date_trans)

[1] "2017-12-18 23:31:01 UTC"

max(data_process$date_trans)

[1] "2017-12-27 23:30:01 UTC"

dim(data_process)

[1] 6965    6

#data_process %>% 
#  filter(date_trans>=as.POSIXct("2017-12-20 23:30:01",tz="UTC"))

#please choose specific time to show
#year_select=2017
#month_select=12
#day_select=22
#df_plot <- data_process %>% 
#  select(date_trans,date,parameter_name,parameter_value,year,month,day) %>% 
#  filter(year==year_select,month==month_select,day==day_select)
#df_plot

# Best to choose
date_choose="2017-12-19"
df_plot2 <- data_process %>% 
  select(date_trans,date,date_filter,parameter_name,parameter_value) %>% 
  filter(date_filter==ymd(date_choose))
df_plot2

summary(df_plot2)

   date_trans                      date            date_filter         parameter_name
 Min.   :2017-12-19 00:00:03   Length:1136        Min.   :2017-12-19   Min.   :500   
 1st Qu.:2017-12-19 00:38:14   Class :character   1st Qu.:2017-12-19   1st Qu.:500   
 Median :2017-12-19 01:15:41   Mode  :character   Median :2017-12-19   Median :500   
 Mean   :2017-12-19 08:22:41                      Mean   :2017-12-19   Mean   :500   
 3rd Qu.:2017-12-19 22:50:21                      3rd Qu.:2017-12-19   3rd Qu.:500   
 Max.   :2017-12-19 23:35:56                      Max.   :2017-12-19   Max.   :500   
 parameter_value
 Min.   :104.0  
 1st Qu.:106.9  
 Median :107.0  
 Mean   :107.0  
 3rd Qu.:107.1  
 Max.   :107.5  

#install.packages('pastecs')
library(pastecs)

Attaching package: ‘pastecs’

The following objects are masked from ‘package:xts’:

    first, last

The following objects are masked from ‘package:dplyr’:

    first, last

The following object is masked from ‘package:tidyr’:

    extract

The following objects are masked from ‘package:data.table’:

    first, last

stat.desc(df_plot2 %>% select(parameter_name,parameter_value))

#install.packages('Hmisc')
library(Hmisc)

Loading required package: lattice
Loading required package: survival
Loading required package: Formula

Attaching package: ‘Hmisc’

The following object is masked from ‘package:psych’:

    describe

The following object is masked from ‘package:plotly’:

    subplot

The following objects are masked from ‘package:dplyr’:

    src, summarize

The following objects are masked from ‘package:base’:

    format.pval, units

describe(df_plot2 %>% select(parameter_name,parameter_value))

df_plot2 %>% select(parameter_name, parameter_value) 

 2  Variables      1136  Observations
----------------------------------------------------------------------------------------
parameter_name 
       n  missing distinct     Info     Mean      Gmd 
    1136        0        1        0      500        0 
               
Value       500
Frequency  1136
Proportion    1
----------------------------------------------------------------------------------------
parameter_value 
       n  missing distinct     Info     Mean      Gmd      .05      .10      .25 
    1136        0      588        1      107   0.2346    106.6    106.7    106.9 
     .50      .75      .90      .95 
   107.0    107.1    107.2    107.3 

lowest : 104.018 104.333 105.644 105.825 106.103, highest: 107.438 107.442 107.453 107.466 107.511
----------------------------------------------------------------------------------------

#install.packages('psych')
library(psych)
describe(df_plot2 %>% select(parameter_name,parameter_value))

df_plot2 %>% select(parameter_name, parameter_value) 

 2  Variables      1136  Observations
----------------------------------------------------------------------------------------
parameter_name 
       n  missing distinct     Info     Mean      Gmd 
    1136        0        1        0      500        0 
               
Value       500
Frequency  1136
Proportion    1
----------------------------------------------------------------------------------------
parameter_value 
       n  missing distinct     Info     Mean      Gmd      .05      .10      .25 
    1136        0      588        1      107   0.2346    106.6    106.7    106.9 
     .50      .75      .90      .95 
   107.0    107.1    107.2    107.3 

lowest : 104.018 104.333 105.644 105.825 106.103, highest: 107.438 107.442 107.453 107.466 107.511
----------------------------------------------------------------------------------------

df_plot2 %>% select(parameter_value) %>% 
  ggplot(aes(y=parameter_value))+
  geom_histogram()

# library(plotly)
# ay <- list(
#   tickfont = list(color = "red"),
#   overlaying = "y",
#   side = "right",
#   title = "second y axis"
# )
# p <- df_plot2 %>% 
#       #arrange(date_trans) %>% 
#       plot_ly(x =~date_trans, y = ~parameter_value,name='value',
#               type='scatter',mode='lines+markers')%>% 
#               add_trace(x=~date_trans, y = ~parameter_name,name='name',
#               type='scatter',mode='lines',yaxis='y2') %>% 
#               layout(yaxis2 = list(overlaying = "y", side = "right")
#               )
# p

#p1<-plot_ly(df_plot2,x =~date_trans, y = ~parameter_value) %>% 
#  add_lines(name='Value') %>% 
#p2<-plot_ly(df_plot2,x =~date_trans, y = ~parameter_name) %>% 
#  add_lines(name='Frequency')
#subplot(p1, p2,nrows=2,shareX = TRUE)

library(plotly)
p1<-plot_ly(df_plot2,x =~date_trans, y = ~parameter_value) %>% 
  add_trace(mode='lines+markers',name='Value')
p2<-plot_ly(df_plot2,x =~date_trans, y = ~parameter_name) %>% 
  add_trace(mode='lines+markers',name='Frequency')

plotly::subplot(p1, p2,nrows=2,shareX = TRUE)

No trace type specified:
  Based on info supplied, a 'scatter' trace seems appropriate.
  Read more about this trace type -> https://plot.ly/r/reference/#scatter
No trace type specified:
  Based on info supplied, a 'scatter' trace seems appropriate.
  Read more about this trace type -> https://plot.ly/r/reference/#scatter

select_data<-df_plot2$parameter_value
Q1 <- quantile(select_data, .25)
Q3 <- quantile(select_data, .75)
IQR <- IQR(select_data)
k=3
df_plot_no_outlier <- subset(df_plot2, select_data> (Q1 - k*IQR) & select_data< (Q3 + k*IQR))

print(dim(df_plot2))

[1] 1136    5

print(dim(df_plot_no_outlier))

[1] 1132    5

p1<-plot_ly(df_plot_no_outlier,x =~date_trans, y = ~parameter_value) %>% 
  add_trace(mode='lines+markers',name='Value')
p2<-plot_ly(df_plot_no_outlier,x =~date_trans, y = ~parameter_name) %>% 
  add_trace(mode='lines+markers',name='Frequency')
p<-plotly::subplot(p1, p2,nrows=2,shareX = TRUE)

No trace type specified:
  Based on info supplied, a 'scatter' trace seems appropriate.
  Read more about this trace type -> https://plot.ly/r/reference/#scatter
No trace type specified:
  Based on info supplied, a 'scatter' trace seems appropriate.
  Read more about this trace type -> https://plot.ly/r/reference/#scatter

p

p1<-plot_ly(df_plot_no_outlier,x =~date_trans, y = ~parameter_value) %>% 
  add_trace(mode='lines+markers',name='Value')
#p2<-plot_ly(df_plot_no_outlier,x =~date_trans, y = ~parameter_name) %>% 
#  add_trace(mode='lines+markers',name='Frequency')
#p<-subplot(p1, p2,nrows=2,shareX = TRUE)
p1

No trace type specified:
  Based on info supplied, a 'scatter' trace seems appropriate.
  Read more about this trace type -> https://plot.ly/r/reference/#scatter
No trace type specified:
  Based on info supplied, a 'scatter' trace seems appropriate.
  Read more about this trace type -> https://plot.ly/r/reference/#scatter

p1<-plot_ly(df_plot_no_outlier,x =~date_trans, y = ~parameter_value) %>% 
  add_trace(mode='lines+markers',name='Value') %>% 
  layout(
    title = "FQC data daily",
    xaxis = list(
      rangeselector = list(
        buttons = list(
          list(
            count = 30,
            label = "min",
            step = "minute",
            stepmode = "backward"),
          list(
            count = 2,
            label = "hour",
            step = "hour",
            stepmode = "backward"),
          list(step = "all"))),

      rangeslider = list(type = "date")),

    yaxis = list(title = "Value"))
p1

No trace type specified:
  Based on info supplied, a 'scatter' trace seems appropriate.
  Read more about this trace type -> https://plot.ly/r/reference/#scatter
No trace type specified:
  Based on info supplied, a 'scatter' trace seems appropriate.
  Read more about this trace type -> https://plot.ly/r/reference/#scatter

# ay <- list(
#   tickfont = list(color = "red"),
#   overlaying = "y",
#   side = "right",
#   title = "second y axis"
# )
# p <- df_plot_no_outlier %>% 
#       arrange(date_trans) %>% 
#       plot_ly(x =~date_trans, y = ~parameter_value,name='value',
#               type='scatter',mode='lines+markers')%>% 
#               add_trace(x=~date_trans, y = ~parameter_name,name='name',
#               type='scatter',mode='lines',yaxis='y2') %>% 
#               layout(yaxis2 = list(overlaying = "y", side = "right")
#               )
# p

num_points=dim(df_plot_no_outlier)[1]
num_points

[1] 1132

USL_value=''
LSL_value=''
LSL<-rep(LSL_value,num_points)
USL<-rep(USL_value,num_points)
p1%>% 
  add_trace(y = ~USL, name = 'USL', mode = 'lines',color="rgba(255, 0, 0, 0.6)") %>% 
  add_trace(y = ~LSL, name = 'LSL', mode = 'lines',color="rgb(20, 20, 20)")

No trace type specified:
  Based on info supplied, a 'scatter' trace seems appropriate.
  Read more about this trace type -> https://plot.ly/r/reference/#scatter
No trace type specified:
  Based on info supplied, a 'bar' trace seems appropriate.
  Read more about this trace type -> https://plot.ly/r/reference/#bar
No trace type specified:
  Based on info supplied, a 'bar' trace seems appropriate.
  Read more about this trace type -> https://plot.ly/r/reference/#bar
minimal value for n is 3, returning requested palette with 3 different levels
minimal value for n is 3, returning requested palette with 3 different levels
Can't display both discrete & non-discrete data on same axis'bar' objects don't have these attributes: 'mode'
Valid attributes include:
'type', 'visible', 'showlegend', 'legendgroup', 'opacity', 'name', 'uid', 'ids', 'customdata', 'meta', 'selectedpoints', 'hoverinfo', 'hoverlabel', 'stream', 'transforms', 'uirevision', 'x', 'x0', 'dx', 'y', 'y0', 'dy', 'text', 'texttemplate', 'hovertext', 'hovertemplate', 'textposition', 'insidetextanchor', 'textangle', 'textfont', 'insidetextfont', 'outsidetextfont', 'constraintext', 'cliponaxis', 'orientation', 'base', 'offset', 'width', 'marker', 'offsetgroup', 'alignmentgroup', 'selected', 'unselected', 'r', 't', '_deprecated', 'error_x', 'error_y', 'xcalendar', 'ycalendar', 'xaxis', 'yaxis', 'idssrc', 'customdatasrc', 'metasrc', 'hoverinfosrc', 'xsrc', 'ysrc', 'textsrc', 'texttemplatesrc', 'hovertextsrc', 'hovertemplatesrc', 'textpositionsrc', 'basesrc', 'offsetsrc', 'widthsrc', 'rsrc', 'tsrc', 'key', 'set', 'frame', 'transforms', '_isNestedKey', '_isSimpleKey', '_isGraticule', '_bbox'
'bar' objects don't have these attributes: 'mode'
Valid attributes include:
'type', 'visible', 'showlegend', 'legendgroup', 'opacity', 'name', 'uid', 'ids', 'customdata', 'meta', 'selectedpoints', 'hoverinfo', 'hoverlabel', 'stream', 'transforms', 'uirevision', 'x', 'x0', 'dx', 'y', 'y0', 'dy', 'text', 'texttemplate', 'hovertext', 'hovertemplate', 'textposition', 'insidetextanchor', 'textangle', 'textfont', 'insidetextfont', 'outsidetextfont', 'constraintext', 'cliponaxis', 'orientation', 'base', 'offset', 'width', 'marker', 'offsetgroup', 'alignmentgroup', 'selected', 'unselected', 'r', 't', '_deprecated', 'error_x', 'error_y', 'xcalendar', 'ycalendar', 'xaxis', 'yaxis', 'idssrc', 'customdatasrc', 'metasrc', 'hoverinfosrc', 'xsrc', 'ysrc', 'textsrc', 'texttemplatesrc', 'hovertextsrc', 'hovertemplatesrc', 'textpositionsrc', 'basesrc', 'offsetsrc', 'widthsrc', 'rsrc', 'tsrc', 'key', 'set', 'frame', 'transforms', '_isNestedKey', '_isSimpleKey', '_isGraticule', '_bbox'
No trace type specified:
  Based on info supplied, a 'scatter' trace seems appropriate.
  Read more about this trace type -> https://plot.ly/r/reference/#scatter
No trace type specified:
  Based on info supplied, a 'bar' trace seems appropriate.
  Read more about this trace type -> https://plot.ly/r/reference/#bar
No trace type specified:
  Based on info supplied, a 'bar' trace seems appropriate.
  Read more about this trace type -> https://plot.ly/r/reference/#bar
minimal value for n is 3, returning requested palette with 3 different levels
minimal value for n is 3, returning requested palette with 3 different levels
Can't display both discrete & non-discrete data on same axis'bar' objects don't have these attributes: 'mode'
Valid attributes include:
'type', 'visible', 'showlegend', 'legendgroup', 'opacity', 'name', 'uid', 'ids', 'customdata', 'meta', 'selectedpoints', 'hoverinfo', 'hoverlabel', 'stream', 'transforms', 'uirevision', 'x', 'x0', 'dx', 'y', 'y0', 'dy', 'text', 'texttemplate', 'hovertext', 'hovertemplate', 'textposition', 'insidetextanchor', 'textangle', 'textfont', 'insidetextfont', 'outsidetextfont', 'constraintext', 'cliponaxis', 'orientation', 'base', 'offset', 'width', 'marker', 'offsetgroup', 'alignmentgroup', 'selected', 'unselected', 'r', 't', '_deprecated', 'error_x', 'error_y', 'xcalendar', 'ycalendar', 'xaxis', 'yaxis', 'idssrc', 'customdatasrc', 'metasrc', 'hoverinfosrc', 'xsrc', 'ysrc', 'textsrc', 'texttemplatesrc', 'hovertextsrc', 'hovertemplatesrc', 'textpositionsrc', 'basesrc', 'offsetsrc', 'widthsrc', 'rsrc', 'tsrc', 'key', 'set', 'frame', 'transforms', '_isNestedKey', '_isSimpleKey', '_isGraticule', '_bbox'
'bar' objects don't have these attributes: 'mode'
Valid attributes include:
'type', 'visible', 'showlegend', 'legendgroup', 'opacity', 'name', 'uid', 'ids', 'customdata', 'meta', 'selectedpoints', 'hoverinfo', 'hoverlabel', 'stream', 'transforms', 'uirevision', 'x', 'x0', 'dx', 'y', 'y0', 'dy', 'text', 'texttemplate', 'hovertext', 'hovertemplate', 'textposition', 'insidetextanchor', 'textangle', 'textfont', 'insidetextfont', 'outsidetextfont', 'constraintext', 'cliponaxis', 'orientation', 'base', 'offset', 'width', 'marker', 'offsetgroup', 'alignmentgroup', 'selected', 'unselected', 'r', 't', '_deprecated', 'error_x', 'error_y', 'xcalendar', 'ycalendar', 'xaxis', 'yaxis', 'idssrc', 'customdatasrc', 'metasrc', 'hoverinfosrc', 'xsrc', 'ysrc', 'textsrc', 'texttemplatesrc', 'hovertextsrc', 'hovertemplatesrc', 'textpositionsrc', 'basesrc', 'offsetsrc', 'widthsrc', 'rsrc', 'tsrc', 'key', 'set', 'frame', 'transforms', '_isNestedKey', '_isSimpleKey', '_isGraticule', '_bbox'

Try another chart: High chart

head(df_plot_no_outlier)

TRY OTHER METHOD:

#install.packages('highcharter')
library(highcharter)

Registered S3 method overwritten by 'quantmod':
  method            from
  as.zoo.data.frame zoo 

dt2%>%
  hchart(type = "line", hcaes(x = V6, y = V66))

Error: Problem with `mutate()` input `x`.
x object 'V6' not found
ℹ Input `x` is `V6`.
Run `rlang::last_error()` to see where the error occurred.

df_plot_no_outlier%>%
  hchart(type = "line", hcaes(x = as.character(date_trans), y = parameter_value))

#install.packages("dygraphs")
library(dygraphs)
df_plot_no_outlier

dt2

Sys.timezone()

[1] "Asia/Ho_Chi_Minh"

tzone(data_time$date_trans)

[1] "UTC"

tzone(df_plot_no_outlier$date_trans)

[1] "UTC"

# Switch to XTS format
data <- xts(x = dt2$V66, 
            order.by = mdy_hms(dt2$V6))
# Add points
p <- dygraph(data) %>%
  dyOptions( drawPoints = TRUE, pointSize = 4,useDataTimezone = TRUE )%>% 
  dyRangeSelector()
p

# Switch to XTS format
data <- xts(x = df_plot_no_outlier$parameter_value, 
            order.by = df_plot_no_outlier$date_trans)
# Add points
p <- dygraph(data) %>%
  dyOptions( drawPoints = TRUE, pointSize = 4,useDataTimezone = TRUE )%>% 
  dyRangeSelector()
p

df_plot_no_outlier

dygraph(df_plot_no_outlier %>% select(date_trans,parameter_value), main = "All", group = "lung-deaths")%>%
  dyOptions( drawPoints = TRUE, pointSize = 4,useDataTimezone = TRUE )

dygraph(df_plot_no_outlier %>% select(date_trans,parameter_name), main = "All", group = "lung-deaths")%>%
  dyOptions( drawPoints = TRUE, pointSize = 4,useDataTimezone = TRUE )

dy_graph <- list(
  dygraph(df_plot_no_outlier %>% select(date_trans,parameter_value), main = "temperature", group = "lung-deaths")%>%
  dyOptions( drawPoints = TRUE, pointSize = 4,useDataTimezone = TRUE ),
dygraph(df_plot_no_outlier %>% select(date_trans,parameter_name), main = "rainfall", group = "lung-deaths")%>%
  dyOptions( drawPoints = TRUE, pointSize = 4,useDataTimezone = TRUE )
)  # end list

# render the dygraphs objects using htmltools
htmltools::browsable(htmltools::tagList(dy_graph))

# Please choose product_type:
file='/media/ad/01D6B57CFBE4DB20/1.Linux/Data/FQC/V04R-V04R-SQLData_3000_head.dat' 
#product_type='31AA015'
product_type_choose='31AA015'
#product_type='31A007G_P'
# Please choose parameter column to show
parameter_column_name='V65'
parameter_column_value='V66'
# Default time column:
product_type_column='V3'
date_column='V6'

savefunc2 <- function(file_path,product_type_column,product_type_choose=NULL,date_column=NULL,
                        parameter_column_name=NULL,parameter_column_value=NULL){
    tryCatch(
      expr    = {
        if ( is.null(product_type_choose)){
          print('not filter')
          dt2 <<- fread(file_path,fill=TRUE,select=c(product_type_column,date_column,parameter_column_name,parameter_column_value))}
        else{
          print('filter')
          dt2 <<- fread(file_path,fill=TRUE,select=c(product_type_column,date_column,parameter_column_name,parameter_column_value)) %>%
          filter(V3 ==product_type_choose)}
        
      },
      warning = function(w){
        cat('Warning: ', w$message, '\n\n');
        n_line2 <- as.numeric(gsub('Stopped early on line (\\d+)\\..*','\\1',w$message))
        if (!is.na(n_line2)) {
          cat('Found ', n_line2,'\n')
          dt2_part1 <- fread(file_path,fill=TRUE, nrows=n_line2-1,select=c(product_type_column,date_column,parameter_column_name,parameter_column_value))
          dt2_part2 <- fread(file_path,fill=TRUE,skip=n_line2,select=c(product_type_column,date_column,parameter_column_name,parameter_column_value))
          if ( is.null(product_type_choose)){
            dt2 <<- rbind(dt2_part1, dt2_part2, fill=T)
          }else{
            dt2 <<- rbind(dt2_part1, dt2_part2, fill=T)%>%
              filter(V3 ==product_type_choose)  
          }
          
        }
      },
      finally = cat("\nFinished. \n")
    );
}


dt<-savefunc2(file,product_type_column,product_type_choose,date_column,parameter_column_name,parameter_column_value)
colnames(dt) <- c("product_type", "date","parameter_freq","parameter_value")
dt<-dt %>%
  mutate( date_trans=mdy_hms(date), # must have mdy_hms for convert date time
          date_filter=as_date(date_trans),
          )
dt

dt<-savefunc2(file,product_type_column)
dt

plotly_chart <- function(data_one_date,check_input_remove_frequency_chart,
                         check_go_data_analyze_date){
  req(check_go_data_analyze_date)
  p1<-plot_ly(data_one_date,x =~date_trans, y = ~parameter_value) %>%
      add_trace(mode='lines+markers',name='Value')
    if (check_input_remove_frequency_chart){
      p1
    } else {
      p2<-plot_ly(data_one_date,x =~date_trans, y = ~parameter_freq) %>%
        add_trace(mode='lines+markers',name='Frequency')
      plotly::subplot(p1, p2,nrows=2,shareX = TRUE)
      
    }
}
plotly_chart(data_one_date(),input$remove_frequency_chart,input$go_data_analyze_date)